<!DOCTYPE HTML>
<html lang="en" >
    <!-- Start book Python数据分析课程讲义 -->
    <head>
        <!-- head:start -->
        <meta charset="UTF-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <title>实战案例：微博情感分析 | Python数据分析课程讲义</title>
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <meta name="description" content="">
        <meta name="generator" content="GitBook 2.6.7">
        <meta name="author" content="BigCat">
        
        <meta name="HandheldFriendly" content="true"/>
        <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
        <meta name="apple-mobile-web-app-capable" content="yes">
        <meta name="apple-mobile-web-app-status-bar-style" content="black">
        <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
        <link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
        
    <link rel="stylesheet" href="../../gitbook/style.css">
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-tbfed-pagefooter/footer.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-splitter/splitter.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-toggle-chapters/toggle.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-highlight/website.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-fontsettings/website.css">
        
    
    

        
    
    
    
    <link rel="prev" href="../../file/part06/6.4.html" />
    

        <!-- head:end -->
    </head>
    <body>
        <!-- body:start -->
        
    <div class="book"
        data-level="5.5"
        data-chapter-title="实战案例：微博情感分析"
        data-filepath="file/part06/6.6.md"
        data-basepath="../.."
        data-revision="Thu Apr 27 2017 00:50:19 GMT+0800 (CST)"
        data-innerlanguage="">
    

<div class="book-summary">
    <nav role="navigation">
        <ul class="summary">
            
            
            
            

            

            
    
        <li class="chapter " data-level="0" data-path="index.html">
            
                
                    <a href="../../index.html">
                
                        <i class="fa fa-check"></i>
                        
                        传智播客Python学院数据分析
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1" data-path="file/part01/1.html">
            
                
                    <a href="../../file/part01/1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.</b>
                        
                        一、工作环境准备及数据分析建模理论基础
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.1" data-path="file/part01/1.1.html">
            
                
                    <a href="../../file/part01/1.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.1.</b>
                        
                        Python 3.x新特性和编码回顾
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="file/part01/1.2.html">
            
                
                    <a href="../../file/part01/1.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.2.</b>
                        
                        DIKW模型与数据工程
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="file/part01/1.3.html">
            
                
                    <a href="../../file/part01/1.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.3.</b>
                        
                        数据分析建模理论基础
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2" data-path="file/part02/2.html">
            
                
                    <a href="../../file/part02/2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.</b>
                        
                        二、科学计算工具NumPy
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.1" data-path="file/part02/2.1.html">
            
                
                    <a href="../../file/part02/2.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.1.</b>
                        
                        ndarray的创建与数据类型
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.2" data-path="file/part02/2.2.html">
            
                
                    <a href="../../file/part02/2.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.2.</b>
                        
                        ndarray的矩阵处理
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.3" data-path="file/part02/2.3.html">
            
                
                    <a href="../../file/part02/2.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.3.</b>
                        
                        ndarray的元素处理
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.4" data-path="file/part02/2.4.html">
            
                
                    <a href="../../file/part02/2.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.4.</b>
                        
                        实战案例：2016美国总统大选民意调查统计
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="3" data-path="file/part03/3.html">
            
                
                    <a href="../../file/part03/3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.</b>
                        
                        三、数据分析工具Pandas
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="3.1" data-path="file/part03/3.1.html">
            
                
                    <a href="../../file/part03/3.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.1.</b>
                        
                        Pandas的数据结构
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.2" data-path="file/part03/3.2.html">
            
                
                    <a href="../../file/part03/3.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.2.</b>
                        
                        Pandas的索引操作
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.3" data-path="file/part03/3.3.html">
            
                
                    <a href="../../file/part03/3.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.3.</b>
                        
                        Pandas的对齐运算
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.4" data-path="file/part03/3.4.html">
            
                
                    <a href="../../file/part03/3.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.4.</b>
                        
                        Pandas的函数应用
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.5" data-path="file/part03/3.5.html">
            
                
                    <a href="../../file/part03/3.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.5.</b>
                        
                        Pandas的层级索引
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.6" data-path="file/part03/3.6.html">
            
                
                    <a href="../../file/part03/3.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.6.</b>
                        
                        Pandas统计计算和描述
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.7" data-path="file/part03/3.7.html">
            
                
                    <a href="../../file/part03/3.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.7.</b>
                        
                        Pandas分组与聚合
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.8" data-path="file/part03/3.8.html">
            
                
                    <a href="../../file/part03/3.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.8.</b>
                        
                        数据清洗、合并、转化和重构
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.9" data-path="file/part03/3.9.html">
            
                
                    <a href="../../file/part03/3.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.9.</b>
                        
                        聚类模型 -- K-Means介绍
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.10" data-path="file/part03/3.10.html">
            
                
                    <a href="../../file/part03/3.10.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.10.</b>
                        
                        实战案例：全球食品数据分析
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="4" data-path="file/part04/4.html">
            
                
                    <a href="../../file/part04/4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.</b>
                        
                        四、数据可视化工具
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="4.1" data-path="file/part04/4.1.html">
            
                
                    <a href="../../file/part04/4.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.1.</b>
                        
                        Matplotlib绘图
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.2" data-path="file/part04/4.2.html">
            
                
                    <a href="../../file/part04/4.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.2.</b>
                        
                        Seaborn绘图
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.3" data-path="file/part04/4.3.html">
            
                
                    <a href="../../file/part04/4.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.3.</b>
                        
                        Bokeh绘图
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.4" data-path="file/part04/4.4.html">
            
                
                    <a href="../../file/part04/4.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.4.</b>
                        
                        实战案例：世界高峰数据可视化
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="5" data-path="file/part06/6.html">
            
                
                    <a href="../../file/part06/6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.</b>
                        
                        五、自然语言处理NLTK
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="5.1" data-path="file/part06/6.1.html">
            
                
                    <a href="../../file/part06/6.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.1.</b>
                        
                        NLTK与自然语言处理基础
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.2" data-path="file/part06/6.2.html">
            
                
                    <a href="../../file/part06/6.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.2.</b>
                        
                        jieba分词
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.3" data-path="file/part06/6.3.html">
            
                
                    <a href="../../file/part06/6.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.3.</b>
                        
                        情感分析
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.4" data-path="file/part06/6.4.html">
            
                
                    <a href="../../file/part06/6.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.4.</b>
                        
                        文本相似度和分类
                    </a>
            
            
        </li>
    
        <li class="chapter active" data-level="5.5" data-path="file/part06/6.6.html">
            
                
                    <a href="../../file/part06/6.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.5.</b>
                        
                        实战案例：微博情感分析
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    


            
            <li class="divider"></li>
            <li>
                <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
                    Published with GitBook
                </a>
            </li>
            
        </ul>
    </nav>
</div>

    <div class="book-body">
        <div class="body-inner">
            <div class="book-header" role="navigation">
    <!-- Actions Left -->
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href="../../" >Python数据分析课程讲义</a>
    </h1>
</div>

            <div class="page-wrapper" tabindex="-1" role="main">
                <div class="page-inner">
                
                
                    <section class="normal" id="section-">
                    
                        <h2 id="&#x5B9E;&#x6218;&#x6848;&#x4F8B;&#xFF1A;&#x5FAE;&#x535A;&#x60C5;&#x611F;&#x5206;&#x6790;">&#x5B9E;&#x6218;&#x6848;&#x4F8B;&#xFF1A;&#x5FAE;&#x535A;&#x60C5;&#x611F;&#x5206;&#x6790;</h2>
<h3 id="&#x6570;&#x636E;&#xFF1A;&#x6BCF;&#x4E2A;&#x6587;&#x672C;&#x6587;&#x4EF6;&#x5305;&#x542B;&#x76F8;&#x5E94;&#x7C7B;&#x7684;&#x6570;&#x636E;">&#x6570;&#x636E;&#xFF1A;&#x6BCF;&#x4E2A;&#x6587;&#x672C;&#x6587;&#x4EF6;&#x5305;&#x542B;&#x76F8;&#x5E94;&#x7C7B;&#x7684;&#x6570;&#x636E;</h3>
<p><img src="../images/weibo_data.png" alt=""></p>
<p><code>0&#xFF1A;&#x559C;&#x60A6;&#xFF1B;1&#xFF1A;&#x6124;&#x6012;&#xFF1B;2&#xFF1A;&#x538C;&#x6076;&#xFF1B;3&#xFF1A;&#x4F4E;&#x843D;</code></p>
<h3 id="&#x6B65;&#x9AA4;">&#x6B65;&#x9AA4;</h3>
<ol>
<li>&#x6587;&#x672C;&#x8BFB;&#x53D6;</li>
<li>&#x5206;&#x5272;&#x8BAD;&#x7EC3;&#x96C6;&#x3001;&#x6D4B;&#x8BD5;&#x96C6;</li>
<li>&#x7279;&#x5F81;&#x63D0;&#x53D6;</li>
<li>&#x6A21;&#x578B;&#x8BAD;&#x7EC3;&#x3001;&#x9884;&#x6D4B;</li>
</ol>
<h3 id="&#x4EE3;&#x7801;&#xFF1A;">&#x4EE3;&#x7801;&#xFF1A;</h3>
<h5 id="toolspy">tools.py</h5>
<pre><code class="lang-python"><span class="hljs-comment"># -*- coding: utf-8 -*-</span>

<span class="hljs-keyword">import</span> re
<span class="hljs-keyword">import</span> jieba.posseg <span class="hljs-keyword">as</span> pseg
<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">import</span> math
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-comment"># &#x52A0;&#x8F7D;&#x5E38;&#x7528;&#x505C;&#x7528;&#x8BCD;</span>
stopwords1 = [line.rstrip() <span class="hljs-keyword">for</span> line <span class="hljs-keyword">in</span> open(<span class="hljs-string">&apos;./&#x4E2D;&#x6587;&#x505C;&#x7528;&#x8BCD;&#x5E93;.txt&apos;</span>, <span class="hljs-string">&apos;r&apos;</span>, encoding=<span class="hljs-string">&apos;utf-8&apos;</span>)]
<span class="hljs-comment"># stopwords2 = [line.rstrip() for line in open(&apos;./&#x54C8;&#x5DE5;&#x5927;&#x505C;&#x7528;&#x8BCD;&#x8868;.txt&apos;, &apos;r&apos;, encoding=&apos;utf-8&apos;)]</span>
<span class="hljs-comment"># stopwords3 = [line.rstrip() for line in open(&apos;./&#x56DB;&#x5DDD;&#x5927;&#x5B66;&#x673A;&#x5668;&#x667A;&#x80FD;&#x5B9E;&#x9A8C;&#x5BA4;&#x505C;&#x7528;&#x8BCD;&#x5E93;.txt&apos;, &apos;r&apos;, encoding=&apos;utf-8&apos;)]</span>
<span class="hljs-comment"># stopwords = stopwords1 + stopwords2 + stopwords3</span>
stopwords = stopwords1


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">proc_text</span><span class="hljs-params">(raw_line)</span>:</span>
    <span class="hljs-string">&quot;&quot;&quot;
        &#x5904;&#x7406;&#x6BCF;&#x884C;&#x7684;&#x6587;&#x672C;&#x6570;&#x636E;
        &#x8FD4;&#x56DE;&#x5206;&#x8BCD;&#x7ED3;&#x679C;
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># 1. &#x4F7F;&#x7528;&#x6B63;&#x5219;&#x8868;&#x8FBE;&#x5F0F;&#x53BB;&#x9664;&#x975E;&#x4E2D;&#x6587;&#x5B57;&#x7B26;</span>
    filter_pattern = re.compile(<span class="hljs-string">&apos;[^\u4E00-\u9FD5]+&apos;</span>)
    chinese_only = filter_pattern.sub(<span class="hljs-string">&apos;&apos;</span>, raw_line)

    <span class="hljs-comment"># 2. &#x7ED3;&#x5DF4;&#x5206;&#x8BCD;+&#x8BCD;&#x6027;&#x6807;&#x6CE8;</span>
    words_lst = pseg.cut(chinese_only)

    <span class="hljs-comment"># 3. &#x53BB;&#x9664;&#x505C;&#x7528;&#x8BCD;</span>
    meaninful_words = []
    <span class="hljs-keyword">for</span> word, flag <span class="hljs-keyword">in</span> words_lst:
        <span class="hljs-comment"># if (word not in stopwords) and (flag == &apos;v&apos;):</span>
            <span class="hljs-comment"># &#x4E5F;&#x53EF;&#x6839;&#x636E;&#x8BCD;&#x6027;&#x53BB;&#x9664;&#x975E;&#x52A8;&#x8BCD;&#x7B49;</span>
        <span class="hljs-keyword">if</span> word <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> stopwords:
            meaninful_words.append(word)

    <span class="hljs-keyword">return</span> <span class="hljs-string">&apos; &apos;</span>.join(meaninful_words)


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">split_train_test</span><span class="hljs-params">(text_df, size=<span class="hljs-number">0.8</span>)</span>:</span>
    <span class="hljs-string">&quot;&quot;&quot;
        &#x5206;&#x5272;&#x8BAD;&#x7EC3;&#x96C6;&#x548C;&#x6D4B;&#x8BD5;&#x96C6;
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># &#x4E3A;&#x4FDD;&#x8BC1;&#x6BCF;&#x4E2A;&#x7C7B;&#x4E2D;&#x7684;&#x6570;&#x636E;&#x80FD;&#x5728;&#x8BAD;&#x7EC3;&#x96C6;&#x4E2D;&#x548C;&#x6D4B;&#x8BD5;&#x96C6;&#x4E2D;&#x7684;&#x6BD4;&#x4F8B;&#x76F8;&#x540C;&#xFF0C;&#x6240;&#x4EE5;&#x9700;&#x8981;&#x4F9D;&#x6B21;&#x5BF9;&#x6BCF;&#x4E2A;&#x7C7B;&#x8FDB;&#x884C;&#x5904;&#x7406;</span>
    train_text_df = pd.DataFrame()
    test_text_df = pd.DataFrame()

    labels = [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>]
    <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> labels:
        <span class="hljs-comment"># &#x627E;&#x51FA;label&#x7684;&#x8BB0;&#x5F55;</span>
        text_df_w_label = text_df[text_df[<span class="hljs-string">&apos;label&apos;</span>] == label]
        <span class="hljs-comment"># &#x91CD;&#x65B0;&#x8BBE;&#x7F6E;&#x7D22;&#x5F15;&#xFF0C;&#x4FDD;&#x8BC1;&#x6BCF;&#x4E2A;&#x7C7B;&#x7684;&#x8BB0;&#x5F55;&#x662F;&#x4ECE;0&#x5F00;&#x59CB;&#x7D22;&#x5F15;&#xFF0C;&#x65B9;&#x4FBF;&#x4E4B;&#x540E;&#x7684;&#x62C6;&#x5206;</span>
        text_df_w_label = text_df_w_label.reset_index()

        <span class="hljs-comment"># &#x9ED8;&#x8BA4;&#x6309;80%&#x8BAD;&#x7EC3;&#x96C6;&#xFF0C;20%&#x6D4B;&#x8BD5;&#x96C6;&#x5206;&#x5272;</span>
        <span class="hljs-comment"># &#x8FD9;&#x91CC;&#x4E3A;&#x4E86;&#x7B80;&#x5316;&#x64CD;&#x4F5C;&#xFF0C;&#x53D6;&#x524D;80%&#x653E;&#x5230;&#x8BAD;&#x7EC3;&#x96C6;&#x4E2D;&#xFF0C;&#x540E;20%&#x653E;&#x5230;&#x6D4B;&#x8BD5;&#x96C6;&#x4E2D;</span>
        <span class="hljs-comment"># &#x5F53;&#x7136;&#x4E5F;&#x53EF;&#x4EE5;&#x968F;&#x673A;&#x62C6;&#x5206;80%&#xFF0C;20%&#xFF08;&#x5C1D;&#x8BD5;&#x5B9E;&#x73B0;&#x4E0B;DataFrame&#x4E2D;&#x7684;&#x968F;&#x673A;&#x62C6;&#x5206;&#xFF09;</span>

        <span class="hljs-comment"># &#x8BE5;&#x7C7B;&#x6570;&#x636E;&#x7684;&#x884C;&#x6570;</span>
        n_lines = text_df_w_label.shape[<span class="hljs-number">0</span>]
        split_line_no = math.floor(n_lines * size)
        text_df_w_label_train = text_df_w_label.iloc[:split_line_no, :]
        text_df_w_label_test = text_df_w_label.iloc[split_line_no:, :]

        <span class="hljs-comment"># &#x653E;&#x5165;&#x6574;&#x4F53;&#x8BAD;&#x7EC3;&#x96C6;&#xFF0C;&#x6D4B;&#x8BD5;&#x96C6;&#x4E2D;</span>
        train_text_df = train_text_df.append(text_df_w_label_train)
        test_text_df = test_text_df.append(text_df_w_label_test)

    train_text_df = train_text_df.reset_index()
    test_text_df = test_text_df.reset_index()
    <span class="hljs-keyword">return</span> train_text_df, test_text_df


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_word_list_from_data</span><span class="hljs-params">(text_df)</span>:</span>
    <span class="hljs-string">&quot;&quot;&quot;
        &#x5C06;&#x6570;&#x636E;&#x96C6;&#x4E2D;&#x7684;&#x5355;&#x8BCD;&#x653E;&#x5165;&#x5230;&#x4E00;&#x4E2A;&#x5217;&#x8868;&#x4E2D;
    &quot;&quot;&quot;</span>
    word_list = []
    <span class="hljs-keyword">for</span> _, r_data <span class="hljs-keyword">in</span> text_df.iterrows():
        word_list += r_data[<span class="hljs-string">&apos;text&apos;</span>].split(<span class="hljs-string">&apos; &apos;</span>)
    <span class="hljs-keyword">return</span> word_list


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">extract_feat_from_data</span><span class="hljs-params">(text_df, text_collection, common_words_freqs)</span>:</span>
    <span class="hljs-string">&quot;&quot;&quot;
        &#x7279;&#x5F81;&#x63D0;&#x53D6;
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># &#x8FD9;&#x91CC;&#x53EA;&#x9009;&#x62E9;TF-IDF&#x7279;&#x5F81;&#x4F5C;&#x4E3A;&#x4F8B;&#x5B50;</span>
    <span class="hljs-comment"># &#x53EF;&#x8003;&#x8651;&#x4F7F;&#x7528;&#x8BCD;&#x9891;&#x6216;&#x5176;&#x4ED6;&#x6587;&#x672C;&#x7279;&#x5F81;&#x4F5C;&#x4E3A;&#x989D;&#x5916;&#x7684;&#x7279;&#x5F81;</span>

    n_sample = text_df.shape[<span class="hljs-number">0</span>]
    n_feat = len(common_words_freqs)
    common_words = [word <span class="hljs-keyword">for</span> word, _ <span class="hljs-keyword">in</span> common_words_freqs]

    <span class="hljs-comment"># &#x521D;&#x59CB;&#x5316;</span>
    X = np.zeros([n_sample, n_feat])
    y = np.zeros(n_sample)

    print(<span class="hljs-string">&apos;&#x63D0;&#x53D6;&#x7279;&#x5F81;...&apos;</span>)
    <span class="hljs-keyword">for</span> i, r_data <span class="hljs-keyword">in</span> text_df.iterrows():
        <span class="hljs-keyword">if</span> (i + <span class="hljs-number">1</span>) % <span class="hljs-number">5000</span> == <span class="hljs-number">0</span>:
            print(<span class="hljs-string">&apos;&#x5DF2;&#x5B8C;&#x6210;{}&#x4E2A;&#x6837;&#x672C;&#x7684;&#x7279;&#x5F81;&#x63D0;&#x53D6;&apos;</span>.format(i + <span class="hljs-number">1</span>))

        text = r_data[<span class="hljs-string">&apos;text&apos;</span>]

        feat_vec = []
        <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> common_words:
            <span class="hljs-keyword">if</span> word <span class="hljs-keyword">in</span> text:
                <span class="hljs-comment"># &#x5982;&#x679C;&#x5728;&#x9AD8;&#x9891;&#x8BCD;&#x4E2D;&#xFF0C;&#x8BA1;&#x7B97;TF-IDF&#x503C;</span>
                tf_idf_val = text_collection.tf_idf(word, text)
            <span class="hljs-keyword">else</span>:
                tf_idf_val = <span class="hljs-number">0</span>

            feat_vec.append(tf_idf_val)

        <span class="hljs-comment"># &#x8D4B;&#x503C;</span>
        X[i, :] = np.array(feat_vec)
        y[i] = int(r_data[<span class="hljs-string">&apos;label&apos;</span>])

    <span class="hljs-keyword">return</span> X, y


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">cal_acc</span><span class="hljs-params">(true_labels, pred_labels)</span>:</span>
    <span class="hljs-string">&quot;&quot;&quot;
        &#x8BA1;&#x7B97;&#x51C6;&#x786E;&#x7387;
    &quot;&quot;&quot;</span>
    n_total = len(true_labels)
    correct_list = [true_labels[i] == pred_labels[i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> range(n_total)]

    acc = sum(correct_list) / n_total
    <span class="hljs-keyword">return</span> acc
</code></pre>
<h4 id="mainpy">main.py</h4>
<pre><code class="lang-python"><span class="hljs-comment"># main.py</span>

<span class="hljs-comment"># -*- coding: utf-8 -*-</span>


<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">import</span> nltk
<span class="hljs-keyword">from</span> tools <span class="hljs-keyword">import</span> proc_text, split_train_test, get_word_list_from_data, \
    extract_feat_from_data, cal_acc
<span class="hljs-keyword">from</span> nltk.text <span class="hljs-keyword">import</span> TextCollection
<span class="hljs-keyword">from</span> sklearn.naive_bayes <span class="hljs-keyword">import</span> GaussianNB

dataset_path = <span class="hljs-string">&apos;./dataset&apos;</span>
text_filenames = [<span class="hljs-string">&apos;0_simplifyweibo.txt&apos;</span>, <span class="hljs-string">&apos;1_simplifyweibo.txt&apos;</span>,
                  <span class="hljs-string">&apos;2_simplifyweibo.txt&apos;</span>, <span class="hljs-string">&apos;3_simplifyweibo.txt&apos;</span>]

<span class="hljs-comment"># &#x539F;&#x59CB;&#x6570;&#x636E;&#x7684;csv&#x6587;&#x4EF6;</span>
output_text_filename = <span class="hljs-string">&apos;raw_weibo_text.csv&apos;</span>

<span class="hljs-comment"># &#x6E05;&#x6D17;&#x597D;&#x7684;&#x6587;&#x672C;&#x6570;&#x636E;&#x6587;&#x4EF6;</span>
output_cln_text_filename = <span class="hljs-string">&apos;clean_weibo_text.csv&apos;</span>

<span class="hljs-comment"># &#x5904;&#x7406;&#x548C;&#x6E05;&#x6D17;&#x6587;&#x672C;&#x6570;&#x636E;&#x7684;&#x65F6;&#x95F4;&#x8F83;&#x957F;&#xFF0C;&#x901A;&#x8FC7;&#x8BBE;&#x7F6E;is_first_run&#x8FDB;&#x884C;&#x914D;&#x7F6E;</span>
<span class="hljs-comment"># &#x5982;&#x679C;&#x662F;&#x7B2C;&#x4E00;&#x6B21;&#x8FD0;&#x884C;&#x9700;&#x8981;&#x5BF9;&#x539F;&#x59CB;&#x6587;&#x672C;&#x6570;&#x636E;&#x8FDB;&#x884C;&#x5904;&#x7406;&#x548C;&#x6E05;&#x6D17;&#xFF0C;&#x9700;&#x8981;&#x8BBE;&#x4E3A;True</span>
<span class="hljs-comment"># &#x5982;&#x679C;&#x4E4B;&#x524D;&#x5DF2;&#x7ECF;&#x5904;&#x7406;&#x4E86;&#x6587;&#x672C;&#x6570;&#x636E;&#xFF0C;&#x5E76;&#x5DF2;&#x7ECF;&#x4FDD;&#x5B58;&#x4E86;&#x6E05;&#x6D17;&#x597D;&#x7684;&#x6587;&#x672C;&#x6570;&#x636E;&#xFF0C;&#x8BBE;&#x4E3A;False&#x5373;&#x53EF;</span>
is_first_run = <span class="hljs-keyword">True</span>


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">read_and_save_to_csv</span><span class="hljs-params">()</span>:</span>
    <span class="hljs-string">&quot;&quot;&quot;
        &#x8BFB;&#x53D6;&#x539F;&#x59CB;&#x6587;&#x672C;&#x6570;&#x636E;&#xFF0C;&#x5C06;&#x6807;&#x7B7E;&#x548C;&#x6587;&#x672C;&#x6570;&#x636E;&#x4FDD;&#x5B58;&#x6210;csv
    &quot;&quot;&quot;</span>

    text_w_label_df_lst = []
    <span class="hljs-keyword">for</span> text_filename <span class="hljs-keyword">in</span> text_filenames:
        text_file = os.path.join(dataset_path, text_filename)

        <span class="hljs-comment"># &#x83B7;&#x53D6;&#x6807;&#x7B7E;&#xFF0C;&#x5373;0, 1, 2, 3</span>
        label = int(text_filename[<span class="hljs-number">0</span>])

        <span class="hljs-comment"># &#x8BFB;&#x53D6;&#x6587;&#x672C;&#x6587;&#x4EF6;</span>
        <span class="hljs-keyword">with</span> open(text_file, <span class="hljs-string">&apos;r&apos;</span>, encoding=<span class="hljs-string">&apos;utf-8&apos;</span>) <span class="hljs-keyword">as</span> f:
            lines = f.read().splitlines()

        labels = [label] * len(lines)

        text_series = pd.Series(lines)
        label_series = pd.Series(labels)

        <span class="hljs-comment"># &#x6784;&#x9020;dataframe</span>
        text_w_label_df = pd.concat([label_series, text_series], axis=<span class="hljs-number">1</span>)
        text_w_label_df_lst.append(text_w_label_df)

    result_df = pd.concat(text_w_label_df_lst, axis=<span class="hljs-number">0</span>)

    <span class="hljs-comment"># &#x4FDD;&#x5B58;&#x6210;csv&#x6587;&#x4EF6;</span>
    result_df.columns = [<span class="hljs-string">&apos;label&apos;</span>, <span class="hljs-string">&apos;text&apos;</span>]
    result_df.to_csv(os.path.join(dataset_path, output_text_filename),
                     index=<span class="hljs-keyword">None</span>, encoding=<span class="hljs-string">&apos;utf-8&apos;</span>)


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">run_main</span><span class="hljs-params">()</span>:</span>
    <span class="hljs-string">&quot;&quot;&quot;
        &#x4E3B;&#x51FD;&#x6570;
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># 1. &#x6570;&#x636E;&#x8BFB;&#x53D6;&#xFF0C;&#x5904;&#x7406;&#xFF0C;&#x6E05;&#x6D17;&#xFF0C;&#x51C6;&#x5907;</span>
    <span class="hljs-keyword">if</span> is_first_run:
        print(<span class="hljs-string">&apos;&#x5904;&#x7406;&#x6E05;&#x6D17;&#x6587;&#x672C;&#x6570;&#x636E;&#x4E2D;...&apos;</span>, end=<span class="hljs-string">&apos; &apos;</span>)
        <span class="hljs-comment"># &#x5982;&#x679C;&#x662F;&#x7B2C;&#x4E00;&#x6B21;&#x8FD0;&#x884C;&#x9700;&#x8981;&#x5BF9;&#x539F;&#x59CB;&#x6587;&#x672C;&#x6570;&#x636E;&#x8FDB;&#x884C;&#x5904;&#x7406;&#x548C;&#x6E05;&#x6D17;</span>

        <span class="hljs-comment"># &#x8BFB;&#x53D6;&#x539F;&#x59CB;&#x6587;&#x672C;&#x6570;&#x636E;&#xFF0C;&#x5C06;&#x6807;&#x7B7E;&#x548C;&#x6587;&#x672C;&#x6570;&#x636E;&#x4FDD;&#x5B58;&#x6210;csv</span>
        read_and_save_to_csv()

        <span class="hljs-comment"># &#x8BFB;&#x53D6;&#x5904;&#x7406;&#x597D;&#x7684;csv&#x6587;&#x4EF6;&#xFF0C;&#x6784;&#x9020;&#x6570;&#x636E;&#x96C6;</span>
        text_df = pd.read_csv(os.path.join(dataset_path, output_text_filename),
                              encoding=<span class="hljs-string">&apos;utf-8&apos;</span>)

        <span class="hljs-comment"># &#x5904;&#x7406;&#x6587;&#x672C;&#x6570;&#x636E;</span>
        text_df[<span class="hljs-string">&apos;text&apos;</span>] = text_df[<span class="hljs-string">&apos;text&apos;</span>].apply(proc_text)

        <span class="hljs-comment"># &#x8FC7;&#x6EE4;&#x7A7A;&#x5B57;&#x7B26;&#x4E32;</span>
        text_df = text_df[text_df[<span class="hljs-string">&apos;text&apos;</span>] != <span class="hljs-string">&apos;&apos;</span>]

        <span class="hljs-comment"># &#x4FDD;&#x5B58;&#x5904;&#x7406;&#x597D;&#x7684;&#x6587;&#x672C;&#x6570;&#x636E;</span>
        text_df.to_csv(os.path.join(dataset_path, output_cln_text_filename),
                       index=<span class="hljs-keyword">None</span>, encoding=<span class="hljs-string">&apos;utf-8&apos;</span>)
        print(<span class="hljs-string">&apos;&#x5B8C;&#x6210;&#xFF0C;&#x5E76;&#x4FDD;&#x5B58;&#x7ED3;&#x679C;&#x3002;&apos;</span>)

    <span class="hljs-comment"># 2. &#x5206;&#x5272;&#x8BAD;&#x7EC3;&#x96C6;&#x3001;&#x6D4B;&#x8BD5;&#x96C6;</span>
    print(<span class="hljs-string">&apos;&#x52A0;&#x8F7D;&#x5904;&#x7406;&#x597D;&#x7684;&#x6587;&#x672C;&#x6570;&#x636E;&apos;</span>)
    clean_text_df = pd.read_csv(os.path.join(dataset_path, output_cln_text_filename),
                                encoding=<span class="hljs-string">&apos;utf-8&apos;</span>)
    <span class="hljs-comment"># &#x5206;&#x5272;&#x8BAD;&#x7EC3;&#x96C6;&#x548C;&#x6D4B;&#x8BD5;&#x96C6;</span>
    train_text_df, test_text_df = split_train_test(clean_text_df)
    <span class="hljs-comment"># &#x67E5;&#x770B;&#x8BAD;&#x7EC3;&#x96C6;&#x6D4B;&#x8BD5;&#x96C6;&#x57FA;&#x672C;&#x4FE1;&#x606F;</span>
    print(<span class="hljs-string">&apos;&#x8BAD;&#x7EC3;&#x96C6;&#x4E2D;&#x5404;&#x7C7B;&#x7684;&#x6570;&#x636E;&#x4E2A;&#x6570;&#xFF1A;&apos;</span>, train_text_df.groupby(<span class="hljs-string">&apos;label&apos;</span>).size())
    print(<span class="hljs-string">&apos;&#x6D4B;&#x8BD5;&#x96C6;&#x4E2D;&#x5404;&#x7C7B;&#x7684;&#x6570;&#x636E;&#x4E2A;&#x6570;&#xFF1A;&apos;</span>, test_text_df.groupby(<span class="hljs-string">&apos;label&apos;</span>).size())

    <span class="hljs-comment"># 3. &#x7279;&#x5F81;&#x63D0;&#x53D6;</span>
    <span class="hljs-comment"># &#x8BA1;&#x7B97;&#x8BCD;&#x9891;</span>
    n_common_words = <span class="hljs-number">200</span>

    <span class="hljs-comment"># &#x5C06;&#x8BAD;&#x7EC3;&#x96C6;&#x4E2D;&#x7684;&#x5355;&#x8BCD;&#x62FF;&#x51FA;&#x6765;&#x7EDF;&#x8BA1;&#x8BCD;&#x9891;</span>
    print(<span class="hljs-string">&apos;&#x7EDF;&#x8BA1;&#x8BCD;&#x9891;...&apos;</span>)
    all_words_in_train = get_word_list_from_data(train_text_df)
    fdisk = nltk.FreqDist(all_words_in_train)
    common_words_freqs = fdisk.most_common(n_common_words)
    print(<span class="hljs-string">&apos;&#x51FA;&#x73B0;&#x6700;&#x591A;&#x7684;{}&#x4E2A;&#x8BCD;&#x662F;&#xFF1A;&apos;</span>.format(n_common_words))
    <span class="hljs-keyword">for</span> word, count <span class="hljs-keyword">in</span> common_words_freqs:
        print(<span class="hljs-string">&apos;{}: {}&#x6B21;&apos;</span>.format(word, count))
    print()

    <span class="hljs-comment"># &#x5728;&#x8BAD;&#x7EC3;&#x96C6;&#x4E0A;&#x63D0;&#x53D6;&#x7279;&#x5F81;</span>
    text_collection = TextCollection(train_text_df[<span class="hljs-string">&apos;text&apos;</span>].values.tolist())
    print(<span class="hljs-string">&apos;&#x8BAD;&#x7EC3;&#x6837;&#x672C;&#x63D0;&#x53D6;&#x7279;&#x5F81;...&apos;</span>, end=<span class="hljs-string">&apos; &apos;</span>)
    train_X, train_y = extract_feat_from_data(train_text_df, text_collection, common_words_freqs)
    print(<span class="hljs-string">&apos;&#x5B8C;&#x6210;&apos;</span>)
    print()

    print(<span class="hljs-string">&apos;&#x6D4B;&#x8BD5;&#x6837;&#x672C;&#x63D0;&#x53D6;&#x7279;&#x5F81;...&apos;</span>, end=<span class="hljs-string">&apos; &apos;</span>)
    test_X, test_y = extract_feat_from_data(test_text_df, text_collection, common_words_freqs)
    print(<span class="hljs-string">&apos;&#x5B8C;&#x6210;&apos;</span>)

    <span class="hljs-comment"># 4. &#x8BAD;&#x7EC3;&#x6A21;&#x578B;Naive Bayes</span>
    print(<span class="hljs-string">&apos;&#x8BAD;&#x7EC3;&#x6A21;&#x578B;...&apos;</span>, end=<span class="hljs-string">&apos; &apos;</span>)
    gnb = GaussianNB()
    gnb.fit(train_X, train_y)
    print(<span class="hljs-string">&apos;&#x5B8C;&#x6210;&apos;</span>)
    print()

    <span class="hljs-comment"># 5. &#x9884;&#x6D4B;</span>
    print(<span class="hljs-string">&apos;&#x6D4B;&#x8BD5;&#x6A21;&#x578B;...&apos;</span>, end=<span class="hljs-string">&apos; &apos;</span>)
    test_pred = gnb.predict(test_X)
    print(<span class="hljs-string">&apos;&#x5B8C;&#x6210;&apos;</span>)

    <span class="hljs-comment"># &#x8F93;&#x51FA;&#x51C6;&#x786E;&#x7387;</span>
    print(<span class="hljs-string">&apos;&#x51C6;&#x786E;&#x7387;&#xFF1A;&apos;</span>, cal_acc(test_y, test_pred))

<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&apos;__main__&apos;</span>:
    run_main()
</code></pre>
<footer class="page-footer"><span class="copyright">Copyright &#xA9; BigCat all right reserved&#xFF0C;powered by Gitbook</span><span class="footer-modification">&#x300C;Revision Time:
2017-04-27 00:48:03&#x300D;
</span></footer>
                    
                    </section>
                
                
                </div>
            </div>
        </div>

        
        <a href="../../file/part06/6.4.html" class="navigation navigation-prev navigation-unique" aria-label="Previous page: 文本相似度和分类"><i class="fa fa-angle-left"></i></a>
        
        
    </div>
</div>

        
<script src="../../gitbook/app.js"></script>

    
    <script src="../../gitbook/plugins/gitbook-plugin-splitter/splitter.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-toggle-chapters/toggle.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-fontsettings/buttons.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-livereload/plugin.js"></script>
    

<script>
require(["gitbook"], function(gitbook) {
    var config = {"disqus":{"shortName":"gitbookuse"},"github":{"url":"https://github.com/dododream"},"search-pro":{"cutWordLib":"nodejieba","defineWord":["gitbook-use"]},"sharing":{"weibo":true,"facebook":true,"twitter":true,"google":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"tbfed-pagefooter":{"copyright":"Copyright © BigCat","modify_label":"「Revision Time:","modify_format":"YYYY-MM-DD HH:mm:ss」"},"baidu":{"token":"ff100361cdce95dd4c8fb96b4009f7bc"},"sitemap":{"hostname":"http://www.treenewbee.top"},"donate":{"wechat":"http://weixin.png","alipay":"http://alipay.png","title":"","button":"赏","alipayText":"支付宝打赏","wechatText":"微信打赏"},"edit-link":{"base":"https://github.com/dododream/edit","label":"Edit This Page"},"splitter":{},"toggle-chapters":{},"highlight":{},"fontsettings":{"theme":"white","family":"sans","size":2},"livereload":{}};
    gitbook.start(config);
});
</script>

        <!-- body:end -->
    </body>
    <!-- End of book Python数据分析课程讲义 -->
</html>
